{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasketch import MinHash\n",
    "#定义了两个句子 已经分好词了 用list存储\n",
    "data1 = ['这个', '程序', '代码', '太乱', '那个', '代码', '规范']\n",
    "data2 = ['这个', '程序', '代码', '不', '规范', '那个', '更', '规范']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'代码', '太乱', '规范', '程序', '那个', '这个'} <class 'set'>\n",
      "{'代码', '不', '规范', '程序', '那个', '更', '这个'} <class 'set'>\n"
     ]
    }
   ],
   "source": [
    "s1 = set(data1)\n",
    "s2 = set(data2)\n",
    "print(s1,type(s1))\n",
    "print(s2,type(s2))#转换成集合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s1和s2的Jaccard相似度实际值 0.625\n"
     ]
    }
   ],
   "source": [
    "actual_jaccard = float( len(s1.intersection(s2)) )/float( len(s1.union(s2)) )#s1和s2 之间的 交比并 就是Jaccard相似度\n",
    "print(\"s1和s2的Jaccard相似度实际值\", actual_jaccard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<datasketch.minhash.MinHash object at 0x7f6f364ec2d0> <class 'datasketch.minhash.MinHash'>\n",
      "<datasketch.minhash.MinHash object at 0x7f6f364ec290> <class 'datasketch.minhash.MinHash'>\n"
     ]
    }
   ],
   "source": [
    "#MinHash转一下\n",
    "m1 = MinHash()\n",
    "m2 = MinHash()\n",
    "for d in data1:\n",
    "    m1.update(d.encode('utf8'))\n",
    "for d in data2:\n",
    "    m2.update(d.encode('utf8'))\n",
    "print(m1,type(m1))\n",
    "print(m2,type(m2))#已经经过MinHash处理了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "使用MinHash预估的Jaccard相似度 0.6015625\n"
     ]
    }
   ],
   "source": [
    "print(\"使用MinHash预估的Jaccard相似度\", m1.jaccard(m2)) #转换前后 Jaccard有偏差 但是不至于很离谱 在大数据处理的情况下 这种做法可以省很多内存"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
